From 1be1e1ccd3d263dd6998788d4f70f01c3b201329 Mon Sep 17 00:00:00 2001 From: "sos22@douglas.cl.cam.ac.uk" Date: Thu, 18 Aug 2005 16:28:41 +0000 Subject: [PATCH] A really nasty workaround for a race. Signed-off-by: Steven Smith, sos22@cam.ac.uk --- .../arch/xen/i386/kernel/process.c | 4 -- .../arch/xen/i386/kernel/smp.c | 10 +---- .../arch/xen/i386/kernel/smpboot.c | 4 +- linux-2.6-xen-sparse/arch/xen/i386/mm/fault.c | 1 - linux-2.6-xen-sparse/arch/xen/kernel/evtchn.c | 16 ++------ linux-2.6-xen-sparse/arch/xen/kernel/reboot.c | 38 +++++++------------ .../include/asm-xen/asm-i386/hypercall.h | 3 ++ xen/arch/x86/domain.c | 15 ++------ xen/arch/x86/mm.c | 15 ++------ xen/arch/x86/time.c | 2 +- xen/common/domain.c | 12 +----- xen/common/event_channel.c | 17 --------- 12 files changed, 33 insertions(+), 104 deletions(-) diff --git a/linux-2.6-xen-sparse/arch/xen/i386/kernel/process.c b/linux-2.6-xen-sparse/arch/xen/i386/kernel/process.c index 39cf60d6e4..12a47fb4e3 100644 --- a/linux-2.6-xen-sparse/arch/xen/i386/kernel/process.c +++ b/linux-2.6-xen-sparse/arch/xen/i386/kernel/process.c @@ -148,8 +148,6 @@ void cpu_idle (void) rmb(); if (cpu_is_offline(cpu)) { - printk("<0>Cpu %d going offline.\n", - cpu); local_irq_disable(); /* Ack it. From this point on until we get woken up, we're not allowed @@ -162,8 +160,6 @@ void cpu_idle (void) #endif play_dead(); local_irq_enable(); - printk("<0>Cpu %d back online.\n", - cpu); } __get_cpu_var(irq_stat).idle_timestamp = jiffies; diff --git a/linux-2.6-xen-sparse/arch/xen/i386/kernel/smp.c b/linux-2.6-xen-sparse/arch/xen/i386/kernel/smp.c index 38c2dfd883..4bb1453592 100644 --- a/linux-2.6-xen-sparse/arch/xen/i386/kernel/smp.c +++ b/linux-2.6-xen-sparse/arch/xen/i386/kernel/smp.c @@ -129,8 +129,6 @@ static inline int __prepare_ICR2 (unsigned int mask) DECLARE_PER_CPU(int, ipi_to_evtchn[NR_IPIS]); -unsigned uber_debug; - static inline void __send_IPI_one(unsigned int cpu, int vector) { unsigned int evtchn; @@ -145,11 +143,7 @@ static inline void __send_IPI_one(unsigned int cpu, int vector) synch_test_bit(evtchn, &s->evtchn_mask[0])) ; #endif - if (uber_debug) - printk("<0>Notifying on evtchn %d.\n", evtchn); - if ((r = notify_via_evtchn(evtchn)) != 0) - printk("<0>Hypervisor stopped us sending an IPI: %d.\n", - r); + notify_via_evtchn(evtchn); } else printk("send_IPI to unbound port %d/%d", cpu, vector); @@ -168,8 +162,6 @@ void __send_IPI_shortcut(unsigned int shortcut, int vector) if (cpu == smp_processor_id()) continue; if (cpu_isset(cpu, cpu_online_map)) { - if (uber_debug) - printk("<0>Sending ipi to %d.\n", cpu); __send_IPI_one(cpu, vector); } } diff --git a/linux-2.6-xen-sparse/arch/xen/i386/kernel/smpboot.c b/linux-2.6-xen-sparse/arch/xen/i386/kernel/smpboot.c index 1fa524b0e8..f47e809e60 100644 --- a/linux-2.6-xen-sparse/arch/xen/i386/kernel/smpboot.c +++ b/linux-2.6-xen-sparse/arch/xen/i386/kernel/smpboot.c @@ -1622,13 +1622,15 @@ DECLARE_PER_CPU(int, timer_irq); void _restore_vcpu(void) { int cpu = smp_processor_id(); + extern atomic_t vcpus_rebooting; + /* We are the first thing the vcpu runs when it comes back, and we are supposed to restore the IPIs and timer interrupts etc. When we return, the vcpu's idle loop will start up again. */ - printk("<0>_restore_vcpu %d.\n", cpu); _bind_virq_to_irq(VIRQ_TIMER, cpu, per_cpu(timer_irq, cpu)); _bind_virq_to_irq(VIRQ_DEBUG, cpu, per_cpu(ldebug_irq, cpu)); _bind_ipi_to_irq(RESCHEDULE_VECTOR, cpu, per_cpu(resched_irq, cpu) ); _bind_ipi_to_irq(CALL_FUNCTION_VECTOR, cpu, per_cpu(callfunc_irq, cpu) ); + atomic_dec(&vcpus_rebooting); } diff --git a/linux-2.6-xen-sparse/arch/xen/i386/mm/fault.c b/linux-2.6-xen-sparse/arch/xen/i386/mm/fault.c index e66a186487..a826b7e38f 100644 --- a/linux-2.6-xen-sparse/arch/xen/i386/mm/fault.c +++ b/linux-2.6-xen-sparse/arch/xen/i386/mm/fault.c @@ -512,7 +512,6 @@ no_context: printk("%08lx\n", regs->eip); dump_fault_path(address); die("Oops", regs, error_code); - while(1); bust_spinlocks(0); do_exit(SIGKILL); diff --git a/linux-2.6-xen-sparse/arch/xen/kernel/evtchn.c b/linux-2.6-xen-sparse/arch/xen/kernel/evtchn.c index 99d1c76ef0..6e3f3357be 100644 --- a/linux-2.6-xen-sparse/arch/xen/kernel/evtchn.c +++ b/linux-2.6-xen-sparse/arch/xen/kernel/evtchn.c @@ -134,8 +134,6 @@ void force_evtchn_callback(void) (void)HYPERVISOR_xen_version(0); } -extern unsigned uber_debug; - /* NB. Interrupts are disabled on entry. */ asmlinkage void evtchn_do_upcall(struct pt_regs *regs) { @@ -147,8 +145,6 @@ asmlinkage void evtchn_do_upcall(struct pt_regs *regs) vcpu_info->evtchn_upcall_pending = 0; - if (uber_debug && cpu != 0) - printk("<0>evtchn_do_upcall on %d.\n", cpu); /* NB. No need for a barrier here -- XCHG is a barrier on x86. */ l1 = xchg(&vcpu_info->evtchn_pending_sel, 0); while ( l1 != 0 ) @@ -162,13 +158,9 @@ asmlinkage void evtchn_do_upcall(struct pt_regs *regs) l2 &= ~(1 << l2i); port = (l1i << 5) + l2i; - if (uber_debug && cpu != 0) - printk("<0>Port %d.\n", port); - if ( (irq = evtchn_to_irq[port]) != -1 ) { - if (uber_debug && cpu != 0) - printk("<0>irq %d.\n", irq); + if ( (irq = evtchn_to_irq[port]) != -1 ) do_IRQ(irq, regs); - } else + else evtchn_device_upcall(port); } } @@ -286,7 +278,7 @@ void _bind_ipi_to_irq(int ipi, int vcpu, int irq) spin_unlock(&irq_mapping_update_lock); - clear_bit(evtchn, HYPERVISOR_shared_info->evtchn_mask); + clear_bit(evtchn, (unsigned long *)HYPERVISOR_shared_info->evtchn_mask); } void _bind_virq_to_irq(int virq, int cpu, int irq) @@ -314,8 +306,6 @@ void _bind_virq_to_irq(int virq, int cpu, int irq) bind_evtchn_to_cpu(evtchn, cpu); spin_unlock(&irq_mapping_update_lock); - - return irq; } int bind_ipi_to_irq(int ipi) diff --git a/linux-2.6-xen-sparse/arch/xen/kernel/reboot.c b/linux-2.6-xen-sparse/arch/xen/kernel/reboot.c index b1e4e697d6..ad6935ea73 100644 --- a/linux-2.6-xen-sparse/arch/xen/kernel/reboot.c +++ b/linux-2.6-xen-sparse/arch/xen/kernel/reboot.c @@ -73,13 +73,14 @@ static void save_vcpu_context(int vcpu, vcpu_guest_context_t *ctxt) BUG_ON(r != 0); gdt_pages = (ctxt->gdt_ents + 511) / 512; ctxt->ctrlreg[3] = machine_to_phys(ctxt->ctrlreg[3]); - for (r = 0; r < gdt_pages; r++) { + for (r = 0; r < gdt_pages; r++) ctxt->gdt_frames[r] = mfn_to_pfn(ctxt->gdt_frames[r]); - } } void _restore_vcpu(int cpu); +atomic_t vcpus_rebooting; + static void restore_vcpu_context(int vcpu, vcpu_guest_context_t *ctxt) { int r; @@ -88,26 +89,27 @@ static void restore_vcpu_context(int vcpu, vcpu_guest_context_t *ctxt) /* This is kind of a hack, and implicitly relies on the fact that the vcpu stops in a place where all of the call clobbered registers are already dead. */ - printk("<0>regs.esp %x.\n", ctxt->user_regs.esp); ctxt->user_regs.esp -= 4; ((unsigned long *)ctxt->user_regs.esp)[0] = ctxt->user_regs.eip; ctxt->user_regs.eip = (unsigned long)_restore_vcpu; ctxt->ctrlreg[3] = phys_to_machine(ctxt->ctrlreg[3]); - for (r = 0; r < gdt_pages; r++) { + for (r = 0; r < gdt_pages; r++) ctxt->gdt_frames[r] = pfn_to_mfn(ctxt->gdt_frames[r]); - } + atomic_set(&vcpus_rebooting, 1); r = HYPERVISOR_boot_vcpu(vcpu, ctxt); if (r != 0) { printk(KERN_EMERG "Failed to reboot vcpu %d (%d)\n", vcpu, r); return; } + /* Hmm... slight hack: make sure the cpus come up in order, + because that way they get the same evtchn numbers this time as + they did last time, which works around a few bugs. */ + /* XXX */ + while (atomic_read(&vcpus_rebooting)) + barrier(); } -/* Whoever decided that printk should call into the scheduler needs to - be taken out and shot */ -#define msg(x) HYPERVISOR_console_io(CONSOLEIO_write, sizeof(x), x) - extern unsigned uber_debug; static int __do_suspend(void *ignore) @@ -223,9 +225,7 @@ static int __do_suspend(void *ignore) smp_suspend(); #endif - msg("xenbus going down.\n"); xenbus_suspend(); - msg("xenbus gone down.\n"); ctrl_if_suspend(); @@ -239,11 +239,9 @@ static int __do_suspend(void *ignore) memcpy(&suspend_record->resume_info, &xen_start_info, sizeof(xen_start_info)); - msg("Suspending...\n"); /* We'll stop somewhere inside this hypercall. When it returns, we'll start resuming after the restore. */ HYPERVISOR_suspend(virt_to_machine(suspend_record) >> PAGE_SHIFT); - msg("Back from suspension\n"); shutting_down = SHUTDOWN_INVALID; @@ -270,9 +268,7 @@ static int __do_suspend(void *ignore) ctrl_if_resume(); - msg("Here comes the xenbus...\n"); xenbus_resume(); - msg("xenbus resumed.\n"); #ifdef CONFIG_SMP smp_resume(); @@ -286,32 +282,26 @@ static int __do_suspend(void *ignore) usbif_resume(); - msg("Restoring cpu contexts...\n"); for (i = 0; i < NR_CPUS; i++) if (cpu_isset(i, feasible_cpus)) restore_vcpu_context(i, &suspended_cpu_records[i]); - msg("All vcpus rebooted.\n"); + printk("<0>All cpus rebooted...\n"); __sti(); out_reenable_cpus: - msg("Reenabling cpus.\n"); while (!cpus_empty(feasible_cpus)) { i = first_cpu(feasible_cpus); - printk("<0>Bring up %d/%d.\n", i, num_online_cpus()); - printk("<0>17 preempt_count %x.\n", preempt_count()); + printk("<0>Bring %d up.\n", i); j = cpu_up(i); - printk("<0>18 preempt_count %x.\n", preempt_count()); + printk("<0>cpu_up(%d) -> %d.\n", i, j); if (j != 0) { printk(KERN_CRIT "Failed to bring cpu %d back up (%d).\n", i, j); err = j; } - printk("<0>%d up.\n", i); cpu_clear(i, feasible_cpus); } - msg("Reenabled cpus.\n"); - uber_debug = 0; out: if ( suspend_record != NULL ) diff --git a/linux-2.6-xen-sparse/include/asm-xen/asm-i386/hypercall.h b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/hypercall.h index 3491eef3c1..cec6e2b9b2 100644 --- a/linux-2.6-xen-sparse/include/asm-xen/asm-i386/hypercall.h +++ b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/hypercall.h @@ -529,6 +529,9 @@ HYPERVISOR_vcpu_down( { int ret; unsigned long ign1; + /* Yes, I really do want to clobber edx here: when we resume a + vcpu after unpickling a multi-processor domain, it returns + here, but clobbers all of the call clobbered registers. */ __asm__ __volatile__ ( TRAP_INSTR : "=a" (ret), "=b" (ign1) diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c index f52c9c5e5f..4f4187dda7 100644 --- a/xen/arch/x86/domain.c +++ b/xen/arch/x86/domain.c @@ -410,10 +410,8 @@ int arch_set_info_guest( if ( !(c->flags & VGCF_VMX_GUEST) ) { if ( ((c->user_regs.cs & 3) == 0) || - ((c->user_regs.ss & 3) == 0) ) { - printf("User regs.cs %x, ss %x.\n", c->user_regs.cs, c->user_regs.ss); + ((c->user_regs.ss & 3) == 0) ) return -EINVAL; - } } clear_bit(_VCPUF_fpu_initialised, &v->vcpu_flags); @@ -458,10 +456,8 @@ int arch_set_info_guest( if ( shadow_mode_refcounts(d) ) { - if ( !get_page(&frame_table[phys_basetab>>PAGE_SHIFT], d) ) { - printf("Bad phys_basetab %lx.\n", phys_basetab); + if ( !get_page(&frame_table[phys_basetab>>PAGE_SHIFT], d) ) return -EINVAL; - } } else { @@ -469,16 +465,13 @@ int arch_set_info_guest( if ( !(c->flags & VGCF_VMX_GUEST) ) #endif if ( !get_page_and_type(&frame_table[phys_basetab>>PAGE_SHIFT], d, - PGT_base_page_table) ) { - printf("Bad phys_basetab2 %lx.\n", phys_basetab); + PGT_base_page_table) ) return -EINVAL; - } } if ( (rc = (int)set_gdt(v, c->gdt_frames, c->gdt_ents)) != 0 ) { put_page_and_type(&frame_table[phys_basetab>>PAGE_SHIFT]); - printf("Failed to set gdt, %d.\n", rc); return rc; } @@ -500,8 +493,6 @@ int arch_set_info_guest( /* Don't redo final setup */ set_bit(_VCPUF_initialised, &v->vcpu_flags); - printf("Arch set_info_guest succeeded.\n"); - return 0; } diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c index a920fadce9..afb9dc482d 100644 --- a/xen/arch/x86/mm.c +++ b/xen/arch/x86/mm.c @@ -2631,24 +2631,17 @@ long set_gdt(struct vcpu *v, int i, nr_pages = (entries + 511) / 512; unsigned long pfn; - if ( entries > FIRST_RESERVED_GDT_ENTRY ) { - printf("Too many entries in gdt (%d).\n", entries); + if ( entries > FIRST_RESERVED_GDT_ENTRY ) return -EINVAL; - } - + shadow_sync_all(d); /* Check the pages in the new GDT. */ for ( i = 0; i < nr_pages; i++ ) { pfn = frames[i]; - if (pfn >= max_page) { - printf("GDT bad as %ld >= %ld.\n", pfn, max_page); + if ((pfn >= max_page) || + !get_page_and_type(&frame_table[pfn], d, PGT_gdt_page) ) goto fail; - } - if (!get_page_and_type(&frame_table[pfn], d, PGT_gdt_page) ) { - printf("Frame %ld looks bad.\n", pfn); - goto fail; - } } /* Tear down the old GDT. */ diff --git a/xen/arch/x86/time.c b/xen/arch/x86/time.c index 657231a86b..81100e7a1c 100644 --- a/xen/arch/x86/time.c +++ b/xen/arch/x86/time.c @@ -831,7 +831,7 @@ static void local_time_calibration(void *unused) tsc_elapsed32 = (u32)tsc_elapsed64; /* tsc_elapsed > stime_elapsed */ - // ASSERT(tsc_elapsed32 != 0); + ASSERT(tsc_elapsed32 != 0); while ( tsc_elapsed32 <= stime_elapsed32 ) { tsc_elapsed32 <<= 1; diff --git a/xen/common/domain.c b/xen/common/domain.c index aee0b90140..b939d90f77 100644 --- a/xen/common/domain.c +++ b/xen/common/domain.c @@ -178,7 +178,6 @@ void domain_shutdown(u8 reason) { struct domain *d = current->domain; struct vcpu *v; - int cntr; if ( d->domain_id == 0 ) { @@ -209,17 +208,8 @@ void domain_shutdown(u8 reason) } /* Put every vcpu to sleep, but don't wait (avoids inter-vcpu deadlock). */ - cntr = 0; - printf("Putting %d to sleep.\n", d->domain_id); - for_each_vcpu ( d, v ) { - if (test_bit(_VCPUF_down, &v->vcpu_flags)) { - printf("vcpu %d is down.\n", v->vcpu_id); - } else { - printf("vcpu %d is up.\n", v->vcpu_id); - } + for_each_vcpu ( d, v ) domain_sleep_nosync(v); - } - printf("Put %d vcpus to sleep for domain shutdown.\n", cntr); } diff --git a/xen/common/event_channel.c b/xen/common/event_channel.c index 8abb035479..21e11a69e4 100644 --- a/xen/common/event_channel.c +++ b/xen/common/event_channel.c @@ -292,8 +292,6 @@ static long evtchn_bind_ipi(evtchn_bind_ipi_t *bind) chn = evtchn_from_port(d, port); chn->state = ECS_IPI; chn->notify_vcpu_id = current->vcpu_id; - printf("Bound ipi on vcpu %d to port %d.\n", current->vcpu_id, - port); } spin_unlock(&d->evtchn_lock); @@ -499,24 +497,9 @@ long evtchn_send(int lport) evtchn_set_pending(rd->vcpu[rchn->notify_vcpu_id], rport); break; case ECS_IPI: - if (current->domain->domain_id != 0) { - printf("Set %d pending on %d.\n", lport, - lchn->notify_vcpu_id); - if (lport == 7) { - struct vcpu *v = ld->vcpu[lchn->notify_vcpu_id]; - struct domain *d = v->domain; - shared_info_t *s = d->shared_info; - printf("pending %x, mask %x, pending_sel %x, upcall_pending %x.\n", - s->evtchn_pending[0], - s->evtchn_mask[0], - v->vcpu_info->evtchn_pending_sel, - v->vcpu_info->evtchn_upcall_pending); - } - } evtchn_set_pending(ld->vcpu[lchn->notify_vcpu_id], lport); break; default: - printf("Failed to set %d pending: state %d.\n", lport, lchn->state); ret = -EINVAL; } -- 2.30.2